
#include "uls.h"
#include "uls.h"
#include <getopt.h>
#include <unistd.h>
#include <limits.h>
#include <time.h>

#define DFL_N_ITERS 0xFFFF

typedef struct _stat_of_round stat_of_round_t;
struct _stat_of_round  {
	int        n_hcodes;
	uls_uint32 hcodes[ULF_MAXDIM_HASHSPACE];
	int   n; // # of buckets
	double avg;
	double sigma2;
	double gamma;
};

char *progname;
char *config_file;
char *target_dir;
char home_dir[1024];
FILE *fp_list;
char *filelist;
char *out_dir;
int  opt_verbose;

uls_lex_t *sam_lex;

uls_kwrd_stat_t *kwrd_stat_list;
int n_kwrd_stat_list;

int target_hashtable_size;
int dim_of_hspace;
int n_iters;

static void usage(void)
{
	err_log("Usage:");
	err_log("\tTo get the ulf-file(*.ulf),");
	err_log("\t\t%s -c <ulc-file> <source1> <source2> ...\n", progname);
	err_log("\tTo specify the target directory of the resultant file(*.ulf),");
	err_log("\t\t%s -d <target-dir> -c <ulc-file> <source1> <source2> ...\n", progname);
	err_log("\tTo process all the filepath in <filelist-file> from start-point <source-directory>,");
	err_log("\t\t%s -c <ulc-file> -l <filelist-file> <source-directory>\n", progname);
	exit(1);
}

static int
parse_options(int argc, char* argv[])
{
	int   opt;
	int   siz;

	out_dir = ".";
	target_dir = NULL;
	n_iters = DFL_N_ITERS;

	while ((opt=getopt(argc, argv, "c:l:d:n:s:vz:")) != -1) {
		switch (opt) {
		case 'c':
			config_file = optarg;
			break;

		case 'l':
			filelist = optarg;
			break;

		case 'n':
			n_iters = atoi(optarg);
			if (n_iters <= 0) n_iters = 1;
			break;

		case 'd':
			out_dir = optarg;
			break;

		case 's':
			if ((siz=atoi(optarg)) > 0) {
				target_hashtable_size =  siz;
			} else {
				err_panic("non-positive hash-size(%d) specified!", siz);
			}
			break;

		case 'z':
			if ((siz=atoi(optarg)) > 0 && siz < ULF_MAXDIM_HASHSPACE) {
				dim_of_hspace =  siz;
			} else {
				err_panic("improper dim-of-hspace(%d) specified!", siz);
			}
			break;

		case 'v':
			++opt_verbose;
			break;
	
		default:
			err_log("undefined option -%c", opt);
			return -1;
			break;
		}
	}

	if (config_file == NULL) {
		err_log("specify the path of config-file(*.ulc)");
		return -1;
	}

	return optind;
}

int
incl_if_keyw(const char* kwrd)
{
	uls_kwrd_stat_t *kwrd_stat;
	int stat = 0;

	kwrd_stat = uls_search_kwrd_stat_sorted(
		kwrd_stat_list, n_kwrd_stat_list, kwrd);

	if (kwrd_stat != NULL) {
		if (kwrd_stat->freq < INT_MAX) {
			++kwrd_stat->freq;
			stat = 1;
		} else {
			stat = -1;
		}
	}

	return stat;
}

int
proc_file(char *fpath)
{
	int tok;

	if (fpath == NULL) return 0;

	uls_push_file(sam_lex, fpath);
	if (opt_verbose)
		err_log("processing %s, ...", fpath);

	for ( ; ; ) {
		tok = uls_gettok(sam_lex);
		if (uls_is_eoi(sam_lex)) break;

		incl_if_keyw(uls_lexeme_text(sam_lex));
	}

	return 1;
}

int
proc_filelist(FILE *fin)
{
	char  linebuff[1024], *lptr, *fpath;
	int   i, len, lno = 0, stat=0;

	while (fgets(linebuff, sizeof(linebuff), fin) != NULL) {
		++lno;

		if ((len=strlen(linebuff))>0 && linebuff[len-1]=='\n') {
			linebuff[--len] = '\0';
		}

		lptr = skip_space(linebuff);
		if (*lptr=='\0' || *lptr=='#') continue;

		fpath = lptr;
		for (i=strlen(fpath)-1; i>=0; i--) {
			if (!isspace(fpath[i])) break;
		}
		fpath[++i] = '\0';

		if (proc_file(fpath) < 0) {
			err_log("fail to process '%s'", fpath);
			stat = -1;
			break;
		}
	}

	return stat;
}

void
go_round(int n_hcodes, uls_uint32* hcodes, int* hash_range, int n_hash_range, stat_of_round_t* p_stat)
{
	uls_kwrd_stat_t *e;
	int i, hash_id, n;
	double sum, sum1, sum2;
	
	memset(hash_range, 0x00, n_hash_range * sizeof(int));

	memcpy(p_stat->hcodes, hcodes, n_hcodes * sizeof(uls_uint32));
	p_stat->n_hcodes = n_hcodes;

	sum = 0.;
	for (i=0; i < n_kwrd_stat_list; i++) {
		e = kwrd_stat_list + i;

		hash_id = ulf_hashfunc(e->kwrd, n_hash_range, -1, n_hcodes, hcodes); 

		if (hash_range[hash_id] < INT_MAX)
			++hash_range[hash_id];

		sum += hash_range[hash_id] * e->freq;
	}

	sum1 = sum2 = 0.;
	n = 0;
	for (hash_id=0; hash_id < n_hash_range; hash_id++) {
		if ((i=hash_range[hash_id]) > 0) {
			printf(" %3d] %d\n", hash_id, i);
			sum1 += i;
			sum2 += i*i;
			++n;
		}
	}

	p_stat->avg = sum1 / n;
	p_stat->sigma2 = sum2 / n - p_stat->avg * p_stat->avg;
	p_stat->n = n;
	p_stat->gamma = sum / n;
}

void
get_random_hcodes(int n_hcodes0, uls_uint32* hcodes0)
{
	uls_uint32 hcode;
	int i, j;

	for (i=0; i<n_hcodes0; i++) {
		hcode = 0;
		for (j=0; j<4; j++) {
			hcode ^= rand();
			hcode <<= 8;
		}
		hcodes0[i] = hcode;
	}
}

void
calc_good_hcode0(int n_samples, int n_hcode0, uls_uint32* hcode0)
{
	int *bucket_size_array;
	stat_of_round_t  best_round_stat, round_stat;
	int i;

	bucket_size_array = (int *) uls_malloc(target_hashtable_size * sizeof(int));

	for (i=0; i<n_samples; i++) {
		get_random_hcodes(n_hcode0, hcode0);
		go_round(n_hcode0, hcode0, bucket_size_array, target_hashtable_size, &round_stat);

		if (opt_verbose && i % 1000 == 0) {
			uls_printf(" avg = %.3f, sigma2 = %.3f, n=%d, gamma=%.3f\n", 
				round_stat.avg, round_stat.sigma2, round_stat.n, round_stat.gamma);
		}

		if (i > 0) {
			if (round_stat.gamma < best_round_stat.gamma) {
				best_round_stat = round_stat;
			}
		} else {
			best_round_stat = round_stat;
		}
	}

	uls_printf("*** avg = %.3f, sigma2 = %.3f, n=%d, gamma=%.3f\n", 
		best_round_stat.avg, best_round_stat.sigma2, best_round_stat.n, best_round_stat.gamma);

	memcpy(hcode0, best_round_stat.hcodes, n_hcode0 * sizeof(uls_uint32));

	uls_printf(" hcode[] = ");
	for (i=0; i<best_round_stat.n_hcodes; i++) {
		uls_printf(" 0x%08X", best_round_stat.hcodes[i]);
	}
	uls_printf("\n");

	uls_mfree(bucket_size_array);
}

static int kwrd_stat_comp_by_freq_reverse(const void* a, const void* b)
{
	uls_kwrd_stat_t *a1 = (uls_kwrd_stat_t *) a;
	uls_kwrd_stat_t *b1 = (uls_kwrd_stat_t *) b;

	return b1->freq - a1->freq;
}

int
main(int argc, char* argv[])
{
	FILE *fp_out;
	char out_file[1024];
	char *ptr0, *ptr;
	int i0, i, len;
	uls_uint32 hcode0[ULF_MAXDIM_HASHSPACE];

	srand(time(NULL));

	progname = uls_filename(argv[0]);
	target_hashtable_size = ULF_HASH_TABLE_SIZE;
	dim_of_hspace = 2;

	if ((i0=parse_options(argc, argv)) < 0) {
		err_log("Incorrect use of command options, try again");
		usage();
	}

	if (filelist != NULL ) {
		if (optind < argc) {
			target_dir = argv[i0++];
		}

		if ((fp_list=fopen(filelist, "r"))==NULL) {
			err_log("%s: fail to read '%s'", __FUNCTION__, filelist);
			return -1;
		}
	}

	ptr0 = uls_filename(config_file);
	// ptr0: filename of config_file

	if ((ptr=strrchr(ptr0, '.'))!=NULL && uls_streql(ptr+1, "ulc")) {
		len = (int) (ptr - ptr0);
	} else {
		err_log("the config-file '%s' must have the suffix 'ulc'", ptr0);
		return -1;
	}

	strcpy(out_file, out_dir);
	i = strlen(out_file);
	out_file[i++] = '/';
	memcpy(out_file+i, ptr0, len);
	i += len;
	strcpy(out_file+i, ".ulf");

	if ((fp_out=fopen(out_file, "w"))==NULL) {
		err_log("%s: fail to create '%s'", __FUNCTION__, out_file);
		if (fp_list) fclose(fp_list);
		return -1;
	}

	if ((sam_lex=uls_create(config_file)) == NULL) {
		err_log("can't init uls-lexer");
		if (fp_list) fclose(fp_list);
		return -1;
	}

	kwrd_stat_list = uls_export_kwtable(sam_lex, &n_kwrd_stat_list);
	if (kwrd_stat_list == NULL) {
		err_log("No keyword infomation!");
		if (fp_list) fclose(fp_list);
		return -1;
	}

	for (i=0; i < n_kwrd_stat_list; i++) {
		kwrd_stat_list[i].freq = 1;
	}

	err_log("Gathering the statistic of keywords usage, ...");

	if (filelist != NULL) {
		getcwd(home_dir, sizeof(home_dir));
		if (target_dir != NULL) {
			if (chdir(target_dir) < 0) {
				err_log("can't change to %s", target_dir);
				free_kwrd_stat_list(kwrd_stat_list);
				uls_destroy(sam_lex);
				fclose(fp_list);
				return -1;
			}
		}

		proc_filelist(fp_list);
		fclose(fp_list);
		chdir(home_dir);

	} else {
		for (i=i0; i<argc; i++) {
			proc_file(argv[i]);
		}
	}

	qsort(kwrd_stat_list, n_kwrd_stat_list, sizeof(uls_kwrd_stat_t),
		kwrd_stat_comp_by_freq_reverse);

	calc_good_hcode0(n_iters, dim_of_hspace, hcode0);

	err_log("Writing the result to %s, ...", out_file);
	ulf_create_file(dim_of_hspace, hcode0, target_hashtable_size, kwrd_stat_list, n_kwrd_stat_list, fp_out);

	free_kwrd_stat_list(kwrd_stat_list);
	uls_destroy(sam_lex);

	return 0;
}
